Node / ReticulumProjects / MeshChatX.git / files / meshchatx / src / backend / recovery / health_monitor.py

Displaying Raw • Download

T8b949e"""Lightweight predictive health monitor.

Runs as a daemon thread on a 5-minute interval, reading in-memory

metrics (log entropy, error rate, memory) and emitting WebSocket

warnings when anomalous trends are detected.

No database queries are made in the monitor loop — all reads come

from in-memory deques kept by PersistentLogHandler and psutil.

"""

Tff7b72import T7ee787asyncio

Tff7b72import T7ee787collections

Tff7b72import T7ee787json

Tff7b72import T7ee787logging

Tff7b72import T7ee787threading

Tff7b72import T7ee787psutil

Te6edf3_log Tff7b72= Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffmeshchatx.health_monitorTa5d6ff"Tb4b4b4)

Tff7b72class T56d364HealthMonitorTb4b4b4:

Te6edf3CHECK_INTERVAL Tff7b72= T79c0ff300 T8b949e# 5 minutes

Te6edf3ENTROPY_WINDOW Tff7b72= T79c0ff6 T8b949e# readings kept (~30 min at default interval)

Te6edf3ENTROPY_WARN_THRESHOLD Tff7b72= T79c0ff1.5 T8b949e# out of max ~2.32 for 5 log levels

Te6edf3ERROR_RATE_WARN Tff7b72= T79c0ff0.3

Te6edf3MEMORY_WARN_MB Tff7b72= T79c0ff100 T8b949e# warn when available < 100 MB

Te6edf3CONSECUTIVE_NEEDED Tff7b72= T79c0ff2 T8b949e# consecutive bad readings before alert

Tff7b72def Tff7b72__init__Tb4b4b4(Tff7b72selfTb4b4b4, Te6edf3log_handlerTb4b4b4, Te6edf3appTff7b72=Tff7b72NoneTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8fflog_handler Tff7b72= Te6edf3log_handler

Tff7b72selfTff7b72.Td2a8ffapp Tff7b72= Te6edf3app

Tff7b72selfTff7b72.Td2a8ff_running Tff7b72= Tff7b72False

Tff7b72selfTff7b72.Td2a8ff_thread Tff7b72= Tff7b72None

Tff7b72selfTff7b72.Td2a8ff_entropy_history Tff7b72= Te6edf3collectionsTff7b72.Td2a8ffdequeTb4b4b4(Te6edf3maxlenTff7b72=Tff7b72selfTff7b72.Td2a8ffENTROPY_WINDOWTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_error_rate_history Tff7b72= Te6edf3collectionsTff7b72.Td2a8ffdequeTb4b4b4(Te6edf3maxlenTff7b72=Tff7b72selfTff7b72.Td2a8ffENTROPY_WINDOWTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_mem_available_history Tff7b72= Te6edf3collectionsTff7b72.Td2a8ffdequeTb4b4b4(Te6edf3maxlenTff7b72=Tff7b72selfTff7b72.Td2a8ffENTROPY_WINDOWTb4b4b4)

Tff7b72def Td2a8ffstartTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72if Tff7b72selfTff7b72.Td2a8ff_runningTb4b4b4:

Tff7b72return

Tff7b72selfTff7b72.Td2a8ff_running Tff7b72= Tff7b72True

Tff7b72selfTff7b72.Td2a8ff_thread Tff7b72= Te6edf3threadingTff7b72.Td2a8ffThreadTb4b4b4(Te6edf3targetTff7b72=Tff7b72selfTff7b72.Td2a8ff_run_loopTb4b4b4, Te6edf3daemonTff7b72=Tff7b72TrueTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_threadTff7b72.Td2a8ffstartTb4b4b4(Tb4b4b4)

Te6edf3_logTff7b72.Td2a8ffinfoTb4b4b4(Ta5d6ff"Ta5d6ffHealthMonitor started (interval=Tffd700%dTa5d6ffs)Ta5d6ff"Tb4b4b4, Tff7b72selfTff7b72.Td2a8ffCHECK_INTERVALTb4b4b4)

Tff7b72def Td2a8ffstopTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8ff_running Tff7b72= Tff7b72False

Tff7b72def Td2a8ff_run_loopTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Te6edf3asyncioTff7b72.Td2a8ffrunTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_async_loopTb4b4b4(Tb4b4b4)Tb4b4b4)

Tff7b72async Tff7b72def Td2a8ff_async_loopTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Tff7b72while Tff7b72selfTff7b72.Td2a8ff_runningTb4b4b4:

Tff7b72tryTb4b4b4:

Tff7b72selfTff7b72.Td2a8ff_checkTb4b4b4(Tb4b4b4)

Tff7b72except Tf85149Exception Tff7b72as Te6edf3excTb4b4b4:

Te6edf3_logTff7b72.Td2a8ffdebugTb4b4b4(Ta5d6ff"Ta5d6ffHealthMonitor check error: Tffd700%sTa5d6ff"Tb4b4b4, Te6edf3excTb4b4b4)

Tff7b72await Te6edf3asyncioTff7b72.Td2a8ffsleepTb4b4b4(Tff7b72selfTff7b72.Td2a8ffCHECK_INTERVALTb4b4b4)

Tff7b72def Td2a8ff_checkTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

Te6edf3entropy Tff7b72= T79c0ff0.0

Te6edf3error_rate Tff7b72= T79c0ff0.0

Tff7b72if Tff7b72selfTff7b72.Td2a8fflog_handlerTb4b4b4:

Te6edf3entropy Tff7b72= Tff7b72selfTff7b72.Td2a8fflog_handlerTff7b72.Td2a8ffcurrent_log_entropy

Te6edf3error_rate Tff7b72= Tff7b72selfTff7b72.Td2a8fflog_handlerTff7b72.Td2a8ffcurrent_error_rate

Te6edf3mem Tff7b72= Te6edf3psutilTff7b72.Td2a8ffvirtual_memoryTb4b4b4(Tb4b4b4)

Te6edf3available_mb Tff7b72= Te6edf3memTff7b72.Td2a8ffavailable Tff7b72/ Tb4b4b4(T79c0ff1024 Tff7b72* T79c0ff1024Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_entropy_historyTff7b72.Td2a8ffappendTb4b4b4(Te6edf3entropyTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_error_rate_historyTff7b72.Td2a8ffappendTb4b4b4(Te6edf3error_rateTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_mem_available_historyTff7b72.Td2a8ffappendTb4b4b4(Te6edf3available_mbTb4b4b4)

Te6edf3warnings Tff7b72= Tb4b4b4[Tb4b4b4]

Tff7b72if Tff7b72selfTff7b72.Td2a8ff_detect_entropy_climbTb4b4b4(Tb4b4b4)Tb4b4b4:

Te6edf3warningsTff7b72.Td2a8ffappendTb4b4b4(

Tb4b4b4{

Ta5d6ff"Ta5d6ffkindTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffentropy_climbingTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffmessageTa5d6ff"Tb4b4b4: Ta5d6fffTa5d6ff"Ta5d6ffLog entropy rising: Tffd700{Te6edf3entropyTffd700:Ta5d6ff.2fTffd700}Ta5d6ff bits (threshold Tffd700{Tff7b72selfTff7b72.Td2a8ffENTROPY_WARN_THRESHOLDTffd700}Ta5d6ff)Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: Tffa657roundTb4b4b4(Te6edf3entropyTb4b4b4, T79c0ff4Tb4b4b4)Tb4b4b4,

Tb4b4b4}Tb4b4b4,

Tb4b4b4)

Tff7b72if Tff7b72selfTff7b72.Td2a8ff_consecutive_aboveTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_error_rate_historyTb4b4b4, Tff7b72selfTff7b72.Td2a8ffERROR_RATE_WARNTb4b4b4)Tb4b4b4:

Te6edf3warningsTff7b72.Td2a8ffappendTb4b4b4(

Tb4b4b4{

Ta5d6ff"Ta5d6ffkindTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6fferror_rate_highTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffmessageTa5d6ff"Tb4b4b4: Ta5d6fffTa5d6ff"Ta5d6ffError rate elevated: Tffd700{Te6edf3error_rateTffd700:Ta5d6ff.0%Tffd700}Ta5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: Tffa657roundTb4b4b4(Te6edf3error_rateTb4b4b4, T79c0ff4Tb4b4b4)Tb4b4b4,

Tb4b4b4}Tb4b4b4,

Tb4b4b4)

Tff7b72if Tff7b72selfTff7b72.Td2a8ff_consecutive_belowTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_mem_available_historyTb4b4b4, Tff7b72selfTff7b72.Td2a8ffMEMORY_WARN_MBTb4b4b4)Tb4b4b4:

Te6edf3warningsTff7b72.Td2a8ffappendTb4b4b4(

Tb4b4b4{

Ta5d6ff"Ta5d6ffkindTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffmemory_lowTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffmessageTa5d6ff"Tb4b4b4: Ta5d6fffTa5d6ff"Ta5d6ffAvailable memory low: Tffd700{Te6edf3available_mbTffd700:Ta5d6ff.0fTffd700}Ta5d6ff MBTa5d6ff"Tb4b4b4,

Ta5d6ff"Ta5d6ffvalueTa5d6ff"Tb4b4b4: Tffa657roundTb4b4b4(Te6edf3available_mbTb4b4b4, T79c0ff1Tb4b4b4)Tb4b4b4,

Tb4b4b4}Tb4b4b4,

Tb4b4b4)

Tff7b72for Te6edf3w Tff7b72in Te6edf3warningsTb4b4b4:

Te6edf3_logTff7b72.Td2a8ffwarningTb4b4b4(Ta5d6ff"Ta5d6ffHealth warning: Tffd700%sTa5d6ff"Tb4b4b4, Te6edf3wTb4b4b4[Ta5d6ff"Ta5d6ffmessageTa5d6ff"Tb4b4b4]Tb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_broadcastTb4b4b4(Te6edf3wTb4b4b4)

Tff7b72def Td2a8ff_detect_entropy_climbTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Return True when entropy climbs in 3+ steps above the warn threshold."""

Te6edf3h Tff7b72= Tff7b72selfTff7b72.Td2a8ff_entropy_history

Tff7b72if Tffa657lenTb4b4b4(Te6edf3hTb4b4b4) Tff7b72< T79c0ff3Tb4b4b4:

Tff7b72return Tff7b72False

Te6edf3vals Tff7b72= Tffa657listTb4b4b4(Te6edf3hTb4b4b4)

Te6edf3last_three Tff7b72= Te6edf3valsTb4b4b4[Tff7b72-T79c0ff3Tb4b4b4:Tb4b4b4]

Tff7b72if Te6edf3last_threeTb4b4b4[Tff7b72-T79c0ff1Tb4b4b4] Tff7b72<Tff7b72= Tff7b72selfTff7b72.Td2a8ffENTROPY_WARN_THRESHOLDTb4b4b4:

Tff7b72return Tff7b72False

Tff7b72return Te6edf3last_threeTb4b4b4[T79c0ff0Tb4b4b4] Tff7b72< Te6edf3last_threeTb4b4b4[T79c0ff1Tb4b4b4] Tff7b72< Te6edf3last_threeTb4b4b4[T79c0ff2Tb4b4b4]

Tf0883e@staticmethod

Tff7b72def Td2a8ff_consecutive_aboveTb4b4b4(Te6edf3deqTb4b4b4, Te6edf3thresholdTb4b4b4)Tb4b4b4:

Tff7b72if Tffa657lenTb4b4b4(Te6edf3deqTb4b4b4) Tff7b72< T79c0ff2Tb4b4b4:

Tff7b72return Tff7b72False

Te6edf3vals Tff7b72= Tffa657listTb4b4b4(Te6edf3deqTb4b4b4)

Tff7b72return Te6edf3valsTb4b4b4[Tff7b72-T79c0ff1Tb4b4b4] Tff7b72> Te6edf3threshold Tff7b72and Te6edf3valsTb4b4b4[Tff7b72-T79c0ff2Tb4b4b4] Tff7b72> Te6edf3threshold

Tf0883e@staticmethod

Tff7b72def Td2a8ff_consecutive_belowTb4b4b4(Te6edf3deqTb4b4b4, Te6edf3thresholdTb4b4b4)Tb4b4b4:

Tff7b72if Tffa657lenTb4b4b4(Te6edf3deqTb4b4b4) Tff7b72< T79c0ff2Tb4b4b4:

Tff7b72return Tff7b72False

Te6edf3vals Tff7b72= Tffa657listTb4b4b4(Te6edf3deqTb4b4b4)

Tff7b72return Te6edf3valsTb4b4b4[Tff7b72-T79c0ff1Tb4b4b4] Tff7b72< Te6edf3threshold Tff7b72and Te6edf3valsTb4b4b4[Tff7b72-T79c0ff2Tb4b4b4] Tff7b72< Te6edf3threshold

Tff7b72def Td2a8ff_broadcastTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3warning_dataTb4b4b4)Tb4b4b4:

Tff7b72if Tff7b72not Tff7b72selfTff7b72.Td2a8ffappTb4b4b4:

Tff7b72return

Tff7b72tryTb4b4b4:

Tff7b72from T7ee787meshchatxT7ee787.T7ee787srcT7ee787.T7ee787backendT7ee787.T7ee787async_utils Tff7b72import Te6edf3AsyncUtils

Te6edf3AsyncUtilsTff7b72.Td2a8ffrun_asyncTb4b4b4(

Tff7b72selfTff7b72.Td2a8ffappTff7b72.Td2a8ffwebsocket_broadcastTb4b4b4(

Te6edf3jsonTff7b72.Td2a8ffdumpsTb4b4b4(Tb4b4b4{Ta5d6ff"Ta5d6fftypeTa5d6ff"Tb4b4b4: Ta5d6ff"Ta5d6ffhealth_warningTa5d6ff"Tb4b4b4, Ta5d6ff"Ta5d6ffdataTa5d6ff"Tb4b4b4: Te6edf3warning_dataTb4b4b4}Tb4b4b4)Tb4b4b4,

Tb4b4b4)Tb4b4b4,

Tb4b4b4)

Tff7b72except Tf85149ExceptionTb4b4b4:

Tff7b72pass

Tf0883e@property

Tff7b72def Td2a8fflatest_snapshotTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:

T8b949e"""Return latest metrics as a dict (useful for diagnostics/tests)."""

Tff7b72return Tb4b4b4{

Ta5d6ff"Ta5d6ffentropyTa5d6ff"Tb4b4b4: Tffa657listTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_entropy_historyTb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6fferror_rateTa5d6ff"Tb4b4b4: Tffa657listTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_error_rate_historyTb4b4b4)Tb4b4b4,

Ta5d6ff"Ta5d6ffmemory_mbTa5d6ff"Tb4b4b4: Tffa657listTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_mem_available_historyTb4b4b4)Tb4b4b4,

Tb4b4b4}